import os

import pandas as pd
import numpy as np
from pyecharts import options as opts
from pyecharts.charts import Bar,Pie,Line
# 目标：真实项目的实战，探索Pandas的数据处理与分析
def analy_website():
    pd.set_option("display.max_columns",None)
    # 读取整个目录，将所有的文件合并到一个dataframe
    data_dir = "../datas/crazyant/blog_access_log"
    df_list = []
    for name in os.listdir(f"{data_dir}"):
        df_list.append(pd.read_csv(f"{data_dir}/{name}", sep=" ", header=None, error_bad_lines=False))
    df = pd.concat(df_list)
    # print(df.head())
    # 取出 ip time status  client
    df = df[[0, 3, 6, 9]].copy()
    # print(df)
    df.columns = ["ip", "stime", "status", "client"]
    print(df.head(5))
    # print(df.dtypes)
    # 2、统计spider的比例¶
    df['isspider'] = df['client'].str.lower().str.contains("spider")
    df_spider = df['isspider'].value_counts()
    print(df_spider.values.tolist())
    bar = (Bar()
           # 添加 X轴 列表数据
           .add_xaxis([str(i) for i in df_spider.index])
           #添加 Y轴数据
           .add_yaxis("是否spider",df_spider.values.tolist())
           .set_global_opts(title_opts=opts.TitleOpts(title="爬虫访问量占比"))
           )
    bar.render("./file/index.html")
    # 3、访问状态码的数量对比
    df_status = df.groupby("status").size()
    pie = (Pie().add("状态码比例",list(zip(df_status.index, df_status)))
           .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}:{c}")))
    pie.render("./file/index2.html")

    # 4、实现按小时、按天粒度的流量统计¶
    #格式化时间
    df["stime"] = pd.to_datetime(df["stime"].str[1:], format="%d/%b/%Y:%H:%M:%S")
    df.set_index("stime",inplace=True)
    df.sort_index(inplace=True)
    print(df)
    # 按小时统计
    df_pvuv = df.resample("H")['ip'].agg(pv=np.size,uv=pd.Series.nunique)
    print(df_pvuv)
    # 按每6个小时统计
    df_pvuv = df.resample("6H")['ip'].agg(pv=np.size,uv=pd.Series.nunique)
    # 按天 统计
    df_pvuv = df.resample("D")['ip'].agg(pv=np.size,uv=pd.Series.nunique)
    print("-----------------------------------------------")
    # print(df_pvuv.index.to_list())
    # print(type(df_pvuv))
    # print(df_pvuv['pv'])
    # 折线图
    line = (
        Line()
            .add_xaxis(df_pvuv.index.to_list())
            .add_yaxis("PV", df_pvuv["pv"].to_list())
            .add_yaxis("UV", df_pvuv["uv"].to_list())
            .set_global_opts(
            title_opts=opts.TitleOpts(title="PVUV数据对比"),
            tooltip_opts=opts.TooltipOpts(trigger="axis", axis_pointer_type="cross")
        )
    )
    line.render("./file/index3.html")

if __name__ == "__main__":
    analy_website()